import pandas as pd
import os
import sys

def process_summary_csv(file_path):
    # 定义列名（使用更简单的列名避免特殊字符问题）
    column_names = [
        'base_dir', 
        'cn_dir', 
        'node_dir', 
        'proc_dir', 
        'comm_type', 
        'real_time', 
        'predict_time', 
        'rel_error'  # 使用简化的列名
    ]
    
    try:
        # 读取CSV文件
        df = pd.read_csv(
            file_path,
            header=None,
            names=column_names,
            quotechar='"',
            skipinitialspace=True
        )
        
        # 清洗数据 - 移除空行或部分空行
        # 1. 删除完全为空的最后一行
        df = df.dropna(how='all')
        
        # 确保数值列是数值类型（解决关键错误）
        df['real_time'] = pd.to_numeric(df['real_time'], errors='coerce')
        df['predict_time'] = pd.to_numeric(df['predict_time'], errors='coerce')
        df['rel_error'] = pd.to_numeric(df['rel_error'], errors='coerce')
        
        # 过滤掉有缺失值的行
        df = df.dropna()
        
        # 按指定字段分组并计算平均值
        grouped = df.groupby(['cn_dir','node_dir', 'proc_dir', 'comm_type'])
        result = grouped.agg({
            'real_time': 'mean',
            'predict_time': 'mean',
            'rel_error': 'mean'
        }).reset_index()
        
        # 重命名结果列为更有意义的名称
        result.rename(columns={
            'real_time': 'avg_total_real_comm_time', 
            'predict_time': 'avg_predict_total_comm_time',
            'rel_error': 'avg_relative_error'
        }, inplace=True)
        
        return result
        
    except Exception as e:
        print(f"Error processing CSV file: {e}")
        # 打印前几行数据帮助调试
        if 'df' in locals():
            print("\nSample data for debugging:")
            print(df.head(3))
            print("\nData types:")
            print(df.dtypes)
        raise

if __name__ == "__main__":
    # 使用原始字符串处理Windows路径
    csv_dir = r"F:\PostGraduate\Point-to-Point-DATA\deal-data-code\C-lop-Prediction\analysis_data\add_program_startup_cost\all_predict_precision"
    
    # 使用os.path.join确保路径正确
    csv_file = os.path.join(csv_dir, "summary.csv")
    avg_precision_csv_dir = os.path.join(csv_dir, "avg_precision_summary.csv")
    
    if not os.path.exists(csv_file):
        print(f"Error: File not found: {csv_file}")
        sys.exit(1)
    
    try:
        # 处理CSV文件并输出结果
        result_df = process_summary_csv(csv_file)
        
        # 保存结果前确保没有空值
        result_df = result_df.dropna()
        
        # 保存结果
        result_df.to_csv(avg_precision_csv_dir, index=False)
        print(f"Successfully saved results to: {avg_precision_csv_dir}")
        
        # 添加验证：读取并显示保存的文件最后几行
        print("\nResults file preview (last 5 rows):")
        preview_df = pd.read_csv(avg_precision_csv_dir)
        print(preview_df.tail(5))
    except Exception as e:
        print(f"Error occurred: {e}")
        sys.exit(1)